#!/bin/bash

europarl=/opt/europarl

for lang in `echo en da fi` ; do 
	# Create empty train and test files
	train=europarl-$lang-train.txt
	test=europarl-$lang-test.txt
	rm -f $train $test $dev
	touch $train $test $dev

	# Process all files
	for f in `ls $europarl/txt/$lang/*.txt` ; do
		# Calculate size of train and test files
		trainsize=`wc -l $train | awk '{ print $1 }'`
		testsize=`wc -l $test | awk '{ print $1 }'`
		echo "Writing $f to $dest (train=$trainsize, test=$testsize)"

		# Select destination (train or test)
		dest=$train
		if [ $(($testsize * 10)) -lt $trainsize ] ; then
			dest=$test
		fi

		# Save file
		cat $f | $europarl/tools/split-sentences.perl -l $lang \
			| /home/matthias/bin/removeXML \
			| $europarl/tools/tokenizer.perl -l $lang >> $dest
	done

	# Shuffle files
	shuf $train -o /tmp/train.$$
	shuf $test -o /tmp/test.$$
	mv /tmp/train.$$ $train
	mv /tmp/test.$$ $test

	# Create subcorpora
	for size in `echo 10000 100000 1000000` ; do 
		cat $train | ./subfile $size > $train.$size
		cat $test | ./subfile $size > $test.$size
	done
done

